# -*- coding: utf-8 -*-
import traceback, datetime, os
import gzip
import base64
import json
from pymongo import Connection
import pycommonlib as pyclib
import workerpool

localPath = os.getcwd()
#conn = Connection('localhost', 27017)
#DB = conn['story']
#coll_tmp = DB['comics_tmp']

SITE_URL = 'http://truyentranh8.com/'
LOCAL_PATH = '/home/data'
FOLDER_STORY = 'doremon'
PREFIX = ''

story  = []

def processJob(item):
    try:
        print 'Process: ', item['name']
        pages = []
        count = 0
        for page in item['images']:
            count = count + 1
            url = page['source']
            result, source, file_name, size = pyclib.saveImage(url, PREFIX, LOCAL_PATH, FOLDER_STORY)
            for j in range(0, 3):
                if result!=None:
                    print 'Count ', count, ': ', url, ' source: ', source
                    pages.append({'source':  source, 'url': url, 'exists': True})
                    break
        if len(pages) > 0:
            doc = {'name': item['name'], 'order': item['order'], 'images': pages}
            story.append(doc)
    except:
        traceback.print_exc()

if __name__ == '__main__':
    try:
        print 'Started process read file'
        print 'Local path: ', localPath
        path_dir = localPath
        
        path_file2 = '{0}/{1}'.format(path_dir, "doremon.data")
        path_file3 = '{0}/{1}'.format(path_dir, "doremon_export.txt")    
        path_file = '{0}/{1}'.format(path_dir, 'doremon.txt')
        print '--- Process file: ', path_file
        
        #f2 = gzip.open(path_file, 'r')
        #text = f2.read()
        #f2.close()
        #text = base64.b64decode(text[:-8])
        # print text
        f2 = open(path_file, 'r')
        text = f2.read()
        f2.close()
        exec("data = " + text)
        
        order = 0
        lstJob = []
        #for i in range(len(data)-1, -1, -1):
        #    item = data[i]
        for item in data: 
            order = order + 1
            item['order'] = order
            lstJob.append(item)
        
        pool = workerpool.WorkerPool(size=10)
        pool.map(processJob, lstJob)
        pool.wait()
        pool.shutdown()
        
        if os.path.isfile(path_file2): 
            print 'File ', path_file2, ' is exists!'
            
        text = json.dumps(story)
        f3 = open(path_file3, 'w')
        f3.write(text)
        f3.close()
        
#        if len(story) > 0:        
#            # Chuyển thành mảng chuối
#            #text = json.dumps(story)
#            text = base64.b64encode(text)
#            text = text + 'hindua88'
#            # Ghi ra file
#            f2 = gzip.open(path_file2, 'wb')
#            f2.write(text)
#            f2.close()
#            print 'File ', path_file2, ' is created!'
        print 'Finished.'
    except:
        traceback.print_exc()
